import  requests
from bs4  import BeautifulSoup
def gethtml(url,headers):
    """
    :param url:
    :param headers:
    :return:
    """

    res= requests.get(url=url,headers=headers).text

    return  res

def parse(data):
    """
    解析网页获取ip数据
    :param data:
    :return:
    """
    result=[]
    soup=BeautifulSoup(data,'lxml')

    iptable=soup.find('table',id='ip_list').find_all('tr')
   # print(iptable)
    for item in iptable:
         try:
             leg=len(item.find_all('td'))
             print(leg)
             if leg!=0:
                 ipaddress = item.find_all('td')[1].get_text()
                 port = item.find_all('td')[2].get_text()
                 address = item.find_all('td')[3].get_text()
                 ip = {
                     'ipaddress': ipaddress,
                     'port': port,
                     'address': address
                 }

                 result.append(ip)
                 ip = None
         except Exception as e:
             print(e)
             continue
    return result


if __name__=='__main__':
    headers={
        'Referer': 'http://www.xicidaili.com/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.110 Safari/537.36'
    }
    url = 'http://www.xicidaili.com'
    result=parse(gethtml(url=url,headers=headers))
